Thirteen* Digital Ways of Looking at -re- in Victorian Poetry

# import software libraries / dependencies
import nltk
import os
import re
import matplotlib.pyplot as plt
from collections import Counter
from nltk.stem import SnowballStemmer

nltk.download('punkt')  # Download NLTK tokenizer data

stemmer = SnowballStemmer("english")  # Initialize SnowballStemmer for English

# create function to preprocess and process files of each directory
def process_directory(corpus_directory):   
    corpus = []

    # Get the directory name as the label
    label = os.path.basename(corpus_directory)

    # Step 1: Get the text of corpus from files
    for filename in os.listdir(corpus_directory):
        with open(os.path.join(corpus_directory, filename), 'r', encoding='utf-8') as file:
            text = file.read()
            corpus.append(text)

    # Step 2: Tokenize the text blob into individual words
    tokenized_corpus = [nltk.word_tokenize(text) for text in corpus]

    # Step 3: Standardize (lower) case, find words that start with re-, stem those words, retain them
    stemmed_corpus = []
    for tokens in tokenized_corpus:
        stemmed_tokens = [stemmer.stem(word.lower()) for word in tokens if re.match(r'\b(re-)\w+', word.lower())]
        stemmed_corpus.append(stemmed_tokens)

    # Step 6: Count the frequency of each re- word
    word_counts = Counter(word for tokens in stemmed_corpus for word in tokens)

    # Step 7: Display the most frequent words
    # most_common_re_words = word_counts.most_common(20)  # Set the desired number of top words
    # for word, count in most_common_re_words:
    #    print(f'The poetry of {label[:-5]} uses {word}: {count} times')

    # Step 8: Plot words and counts in a bar chart
    plt.figure(figsize=(10, 5))
    top_words, top_counts = zip(*word_counts.most_common(20))
    plt.bar(top_words, top_counts)
    plt.title(f'Frequency of Stemmed Re- Words in {label[:-5].capitalize()}\'s Poetry')
    plt.xticks(rotation=65)
    plt.show()

# Directories to process
corpus_directories = [
    '/home/adammazel/Documents/Digital_Scholarship/re-victorian-poetry/cta/swinburne/swinburne_noBP',
    '/home/adammazel/Documents/Digital_Scholarship/re-victorian-poetry/cta/hardy/hardy_noBP',
    '/home/adammazel/Documents/Digital_Scholarship/re-victorian-poetry/cta/field/field_NoBP',
    '/home/adammazel/Documents/Digital_Scholarship/re-victorian-poetry/cta/rossetti_dg/rossetti_dg_NoBP',
    # Add more directories here
]

# Process each directory
for directory in corpus_directories:
    process_directory(directory)

# import software dependencies / libraries
import nltk
import os
import re
from nltk.corpus import stopwords
from nltk.util import ngrams
from collections import Counter
import string
from nltk.stem import SnowballStemmer

nltk.download('stopwords')

# Directories to process, Aliases 
corpus_directories = {
    '/home/adammazel/Documents/Digital_Scholarship/re-victorian-poetry/cta/swinburne/swinburne_noBP': "AC Swinburne",
    '/home/adammazel/Documents/Digital_Scholarship/re-victorian-poetry/cta/hardy/hardy_noBP': "Thomas Hardy",
    '/home/adammazel/Documents/Digital_Scholarship/re-victorian-poetry/cta/field/field_NoBP': "Michael Field",
    '/home/adammazel/Documents/Digital_Scholarship/re-victorian-poetry/cta/rossetti_dg/rossetti_dg_NoBP': "DG Rossetti",
}

# RegEx to match words starting with "re-"
re_word_pattern = r'\b(re-\w+)'

# Dictionary to store directory/poet: bigram frequencies 
bigram_frequencies = {alias: Counter() for alias in corpus_directories.values()}

# Define the punctuation to remove (including curly quotation marks)
punctuation_to_remove = string.punctuation.replace('-', '') + "‘’“”"

# Preprocess and Process each directory
for corpus_directory, alias in corpus_directories.items():
    # Get the text from the text files in the corpus
    for filename in os.listdir(corpus_directory):
        if filename.endswith('.txt'):
            with open(os.path.join(corpus_directory, filename), 'r', encoding='utf-8') as file:
                text = file.read()

                # Tokenize the text
                tokens = nltk.word_tokenize(text)

                # Remove punctuation except hyphens and standardize case
                translator = str.maketrans('', '', punctuation_to_remove)
                preprocessed_text = ' '.join(tokens).translate(translator).lower()

                # Remove stopwords
                stop_words = set(stopwords.words("english"))
                filtered_tokens = [word for word in preprocessed_text.split() if word not in stop_words]

                # Find bigrams
                bigrams = list(ngrams(filtered_tokens, 2))

                # Filter bigrams to include only those starting with "re-"
                re_bigrams = [bigram for bigram in bigrams if re.match(re_word_pattern, bigram[0])]

                # Count the bigrams
                bigram_frequency = Counter(re_bigrams)

                # Add the bigram frequencies as values of the current directory/poet as keys
                bigram_frequencies[alias].update(bigram_frequency)

# Print the sorted bigram frequencies per directory/poet
for alias, frequencies in bigram_frequencies.items():
    print(f"{alias}:")
    for bigram, frequency in sorted(frequencies.items(), key=lambda x: x[0]):  # Sort alphabetically
        print(f"{bigram}: {frequency}")
    print()

# Import software libraries / dependencies
import nltk
import os
import re
import matplotlib.pyplot as plt
import string

nltk.download('punkt')  # Download NLTK tokenizer data

# Define a function to count words that start with "re-"
def count_re_words(text):
    words = nltk.word_tokenize(text)
    return sum(1 for word in words if word.lower().startswith("re-"))

# Define a function to remove all punctuation except hyphens
def remove_punctuation_except_hyphens(text):
    translator = str.maketrans('', '', string.punctuation.replace('-', ''))
    return text.translate(translator)

# Specify the parent directory containing multiple text file directories
parent_directory = '/home/adammazel/Documents/Digital_Scholarship/re-victorian-poetry/cta/'

# Soecific irectories to process
all_directory_names = [
    'swinburne/swinburne_noBP',
    'hardy/hardy_noBP',
    'field/field_NoBP',
    'rossetti_dg/rossetti_dg_NoBP',
]

# Initialize a dictionary to store year-wise percentages
year_percentages = {}

# Process each directory
for directory_name in all_directory_names:
    # Construct the full path to the current directory
    text_files_directory = os.path.join(parent_directory, directory_name)

    if os.path.isdir(text_files_directory):
        # Initialize a dictionary to store year-wise percentages for the current directory
        directory_percentages = {}

        # Initialize a dictionary to store file names for the current directory
        file_names = {}

        # Process each text file in the current directory
        for filename in os.listdir(text_files_directory):
            if filename.endswith('.txt'):
                # Extract the year from the filename using a regular expression
                year_match = re.match(r'(\d{4})_', filename)
                if year_match:
                    year = int(year_match.group(1))
                    with open(os.path.join(text_files_directory, filename), 'r', encoding='utf-8') as file:
                        text = file.read()
                        # Remove punctuation except hyphens
                        text = remove_punctuation_except_hyphens(text)
                        # normalize counts
                        total_words = len(nltk.word_tokenize(text))
                        re_word_count = count_re_words(text)
                        percentage_re_words = (re_word_count / total_words) * 100
                        
                        #add to dictionary: values: counts to keys: years
                        directory_percentages[year] = percentage_re_words
                        
                        # Extract the text between underscores in the filename
                        file_name_parts = filename.split('_')
                        if len(file_name_parts) > 2:
                            file_name = '_'.join(file_name_parts[1:-1])
                        else:
                            file_name = file_name_parts[1]
                        
                        #add to dictionary: values: filenames to keys: years 
                        file_names[year] = file_name  # Store the extracted file name

        # Sort the dictionary by keys (years) for the current directory
        sorted_directory_percentages = {year: directory_percentages[year] for year in sorted(directory_percentages)}

        # Store the results for the current directory
        year_percentages[directory_name] = {
            'percentages': sorted_directory_percentages,
            'file_names': file_names  # Store file names for this directory
        }

# Step 2: Plot the keys (years) and values (percentages) in a line graph for each directory
plt.figure(figsize=(15, 8))

for directory_name, data in year_percentages.items():
    percentages = data['percentages']
    file_names = data['file_names']
    years = list(percentages.keys())
    percentages = list(percentages.values())
    
    # Plot the data points
    plt.plot(years, percentages, marker='o', linestyle='-', label=directory_name[:-5])
    
    # Add annotations for each data point if the value is greater than 0
    for year, percentage in zip(years, percentages):
        if percentage > 0:
            file_name = file_names[year]
            annotation = f"{file_name}"
            plt.annotate(annotation, (year, percentage), textcoords="offset points", xytext=(0, 10), ha='center')

plt.ylim(0, 0.0525)
plt.xlabel('Year')
plt.ylabel('Percentage of Words Starting with "Re-"')
plt.title('When Were "Re-" Words Used in Field\'s, Hardy\'s, DG Rossetti\'s, and Swinburne\'s Poetry?')
plt.xticks(rotation=45)
plt.grid(True)
plt.legend()  # Show legend indicating directory names

# Display the line graph
plt.tight_layout()
plt.show()

# Import software libraries / dependencies
import nltk
from nltk.sentiment.vader import SentimentIntensityAnalyzer
import os
import re
import matplotlib.pyplot as plt

# Initialize the VADER sentiment analyzer
sia = SentimentIntensityAnalyzer()

# Function to calculate aggregate sentiment for a collection of sentences
def calculate_aggregate_sentiment(sentences):
    positive_score = 0
    negative_score = 0
    neutral_score = 0
    total_sentences = 0

    for sentence in sentences:
        sentiment = sia.polarity_scores(sentence)
        positive_score += sentiment['pos']
        negative_score += sentiment['neg']
        neutral_score += sentiment['neu']
        total_sentences += 1

    if total_sentences > 0:
        avg_positive_score = positive_score / total_sentences
        avg_negative_score = negative_score / total_sentences
        avg_neutral_score = neutral_score / total_sentences

        if avg_positive_score > avg_negative_score:
            overall_sentiment = "Positive"
        elif avg_positive_score < avg_negative_score:
            overall_sentiment = "Negative"
        else:
            overall_sentiment = "Neutral"

        return {
            "Total Sentences Analyzed": total_sentences,
            "Average Positive Score": avg_positive_score,
            "Average Negative Score": avg_negative_score,
            "Average Neutral Score": avg_neutral_score,
            "Overall Sentiment": overall_sentiment,
        }

# Specify the directories you want to process with aliases
corpus_directories = {
    '/home/adammazel/Documents/Digital_Scholarship/re-victorian-poetry/cta/swinburne/swinburne_noBP': "AC Swinburne",
    '/home/adammazel/Documents/Digital_Scholarship/re-victorian-poetry/cta/hardy/hardy_noBP': "Thomas Hardy",
    '/home/adammazel/Documents/Digital_Scholarship/re-victorian-poetry/cta/field/field_NoBP': "Michael Field",
    '/home/adammazel/Documents/Digital_Scholarship/re-victorian-poetry/cta/rossetti_dg/rossetti_dg_NoBP': "DG Rossetti",
    # Add more directories here
}

# Create lists to store poet names and their corresponding score differences
poet_names = []
score_differences = []

# Process each directory
for corpus_directory, alias in corpus_directories.items():
    sentences = []

    # Read and preprocess the text files in the corpus
    for filename in os.listdir(corpus_directory):
        if filename.endswith('.txt'):
            with open(os.path.join(corpus_directory, filename), 'r', encoding='utf-8') as file:
                text = file.read()
                sentences += nltk.sent_tokenize(text)

    # Calculate aggregate sentiment
    results = calculate_aggregate_sentiment(sentences)

    # Calculate the difference between positive and negative scores
    positive_score = results["Average Positive Score"]
    negative_score = results["Average Negative Score"]
    score_difference = positive_score - negative_score

    poet_names.append(alias)
    score_differences.append(score_difference)

# Sort the poet names and score differences by score differences in descending order
sorted_poet_names, sorted_score_differences = zip(*sorted(zip(poet_names, score_differences), key=lambda x: x[1], reverse=True))

# Create a bar chart of the score differences with poet names on the x-axis
plt.figure(figsize=(10, 6))
plt.bar(sorted_poet_names, sorted_score_differences, color='skyblue')
plt.xlabel("Poet")
plt.ylabel('Difference between Positive and Negative Scores')
plt.title('Whose Poetry Is Most Positive?')
plt.xticks(rotation=45, ha='right')
plt.grid(axis='y')
plt.show()

import nltk
from nltk.sentiment.vader import SentimentIntensityAnalyzer
import os

# Initialize the VADER sentiment analyzer
sia = SentimentIntensityAnalyzer()

# Function to calculate aggregate sentiment for a collection of sentences
def calculate_aggregate_sentiment(sentences):
    positive_score = 0
    negative_score = 0
    neutral_score = 0
    total_sentences = 0

    for sentence in sentences:
        sentiment = sia.polarity_scores(sentence)
        positive_score += sentiment['pos']
        negative_score += sentiment['neg']
        neutral_score += sentiment['neu']
        total_sentences += 1

    if total_sentences > 0:
        avg_positive_score = positive_score / total_sentences
        avg_negative_score = negative_score / total_sentences
        avg_neutral_score = neutral_score / total_sentences

        if avg_positive_score > avg_negative_score:
            overall_sentiment = "Positive"
        elif avg_positive_score < avg_negative_score:
            overall_sentiment = "Negative"
        else:
            overall_sentiment = "Neutral"

        return {
            "Total Sentences Analyzed": total_sentences,
            "Average Positive Score": avg_positive_score,
            "Average Negative Score": avg_negative_score,
            "Average Neutral Score": avg_neutral_score,
            "Overall Sentiment": overall_sentiment,
        }
    else:
        return {"No sentences with 're-' words found for analysis."}

# Specify the directories you want to process with aliases
corpus_directories = {
 
        '/home/adammazel/Documents/Digital_Scholarship/re-victorian-poetry/cta/swinburne/swinburne_noBP': "AC Swinburne",
    '/home/adammazel/Documents/Digital_Scholarship/re-victorian-poetry/cta/hardy/hardy_noBP': "Thomas Hardy",
    '/home/adammazel/Documents/Digital_Scholarship/re-victorian-poetry/cta/field/field_NoBP': "Michael Field",
    '/home/adammazel/Documents/Digital_Scholarship/re-victorian-poetry/cta/rossetti_dg/rossetti_dg_NoBP': "DG Rossetti",
       # Add more directories here
}

# Process each directory
for corpus_directory, alias in corpus_directories.items():
    sentences = []

    # Read and preprocess the text files in the corpus
    for filename in os.listdir(corpus_directory):
        if filename.endswith('.txt'):
            with open(os.path.join(corpus_directory, filename), 'r', encoding='utf-8') as file:
                text = file.read()
                sentences += nltk.sent_tokenize(text)

    # Filter sentences with words starting with "re-"
    sentences_with_re = [sentence for sentence in sentences if any(word.lower().startswith("re-") for word in nltk.word_tokenize(sentence))]

    # Calculate aggregate sentiment for the filtered sentences
    results = calculate_aggregate_sentiment(sentences_with_re)

    # Print results for the current directory
    print(f"{alias}")
    for key, value in results.items():
        print(f"{key}: {value}")
    print()